import os
import time
import json
import concurrent.futures
from threading import Lock

import requests
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.service import Service
from openpyxl import Workbook, load_workbook
# 创建锁对象用于保护文件操作
excel_lock = Lock()


#获取代理服务器地址端口，如果没有则不需要管
def get_ip():
    """获取可用的代理IP，直到成功为止"""

    def test_ip(ip):
        try:
            proxies = {
                'http': f'http://{ip}',
                'https': f'http://{ip}'
            }
            response = requests.get('http://www.baidu.com',
                                    proxies=proxies,
                                    timeout=10)
            return response.status_code == 200
        except:
            return False

    while True:
        try:
            api = '' #输入自己的代理服务器的API，如果没有则不用管
            ip = requests.get(api).text.strip()

            if test_ip(ip):
                print(f"找到可用代理IP: {ip}")
                return ip
            else:
                print(f"代理IP {ip} 不可用，继续尝试...")
                time.sleep(2)  # 避免频繁请求

        except Exception as e:
            print(f"获取代理IP时出错: {e}")
            time.sleep(2)  # 出错时等待后重试

def save_score_data(file_name,school_id, province, year, subject, score_data):
    """保存数据到Excel（线程安全版本）"""
    with excel_lock:  # 使用锁保护文件操作
        try:
            excel_dir = r'C:\school\score_data'#保存文件的地址
            if not os.path.exists(excel_dir):
                os.makedirs(excel_dir)

            excel_path = os.path.join(excel_dir, f'{file_name}_score.xlsx')

            # 如果文件不存在，创建新文件并添加表头
            if not os.path.exists(excel_path):
                wb = Workbook()
                ws = wb.active
                ws.title = "分数线数据"
                headers = ["学校ID", "省份", "年份", "分数线数据"]
                for col, header in enumerate(headers, 1):
                    ws.cell(row=1, column=col, value=header)
            else:
                wb = load_workbook(excel_path)
                ws = wb.active

            # 获取下一个可用行
            next_row = ws.max_row + 1

            # 添加新数据
            ws.cell(row=next_row, column=1, value=school_id)
            ws.cell(row=next_row, column=2, value=province)
            ws.cell(row=next_row, column=3, value=year)
            ws.cell(row=next_row, column=4, value=subject)
            ws.cell(row=next_row, column=5, value=str(score_data))

            # 保存文件
            wb.save(excel_path)
            print(f"数据已保存到第 {next_row} 行")

        except Exception as e:
            print(f"保存数据时出错: {e}")





def save_failed_data(file_name,school_id, province, year, subject):
    """保存爬取失败的数据记录（线程安全版本）"""
    with excel_lock:  # 使用锁保护文件操作
        try:
            excel_dir = r'C:\school\score_data'
            if not os.path.exists(excel_dir):
                os.makedirs(excel_dir)

            excel_path = os.path.join(excel_dir, f'{file_name}_failed.xlsx')

            # 如果文件不存在，创建新文件并添加表头
            if not os.path.exists(excel_path):
                wb = Workbook()
                ws = wb.active
                ws.title = "失败记录"
                headers = ["学校ID", "省份", "年份","科目", "失败原因"]
                for col, header in enumerate(headers, 1):
                    ws.cell(row=1, column=col, value=header)
            else:
                wb = load_workbook(excel_path)
                ws = wb.active

            # 获取下一个可用行
            next_row = ws.max_row + 1

            # 添加新数据
            ws.cell(row=next_row, column=1, value=school_id)  # 学校ID
            ws.cell(row=next_row, column=2, value=province)
            ws.cell(row=next_row, column=3, value=year)
            ws.cell(row=next_row, column=4, value=subject)
            ws.cell(row=next_row, column=5, value="数据获取失败")

            # 保存文件
            wb.save(excel_path)
            print(f"失败记录已保存到第 {next_row} 行")

        except Exception as e:
            print(f"保存失败记录时出错: {e}")


def load_cookies(driver, cookie_file):
    """加载cookies"""
    try:
        with open(cookie_file, 'r', encoding='utf-8') as f:
            cookies = json.loads(f.read())
        for cookie in cookies:
            driver.add_cookie(cookie)
        return True
    except Exception as e:
        print(f"加载cookies失败：{str(e)}")
        return False


import re

#该方法用于获取类型列表（理科、文科、综合、物理类、历史类）
def get_subject_types(type_list):
    type_lists = re.findall(r'[\u4e00-\u9fa5]+', type_list)
    return type_lists



import random
def process_school(school):
    school_id = school[0]
    school_name = school[1]

    try:
        # 初始化浏览器
        service = Service(executable_path='D:\chromedriver_win32 (1)\chromedriver.exe')#输入自己的浏览器驱动地址

        #ip=get_ip()#上面如果输入了代理服务器API则将这行和下面第二行注释取消
        chrome_options=webdriver.ChromeOptions()
        #ip=get_ip()#上面如果输入了代理服务器API则将这行和下面第二行注释取消


        # 创建 Chrome 浏览器实例
        driver = webdriver.Chrome(service=service, options=chrome_options)
        # 访问网页并加载cookies
        url = f'https://www.gaokao.cn/school/{school_id}/provinceline'
        driver.get(url)
        try:
            buttom_xpath='/html/body/div[2]/div/div[2]/div/div[2]/div[3]/button'
            parent=WebDriverWait(driver, 5).until(EC.visibility_of_element_located((By.XPATH, "/html/body/div[2]/div/div[2]/div/div[2]")))
            button=WebDriverWait(driver,5).until(EC.element_to_be_clickable((By.XPATH, buttom_xpath)))
            button.click()
        except Exception as e:
            print("wu")
        driver.maximize_window()
        if not load_cookies(driver, 'cookies_juejin.json'):
            return

        driver.refresh()
        time.sleep(10)
        driver.execute_script("window.scrollBy(0, 300);")
        time.sleep(2)

        # 遍历省份
        # 获取省份

        province_dropdown = driver.find_elements(By.CLASS_NAME, "ant-select-selection--single")[0]
        province_dropdown.click()
        province_opt=WebDriverWait(driver,5).until(EC.visibility_of_element_located((By.XPATH,'//*[@id="proline"]/div/div[1]/div/div[1]/div[2]/div/div/div')))
        province_list = driver.find_elements(By.XPATH, '//*[@id="proline"]/div/div[1]/div/div[1]/div[2]/div/div/div')[0].text
        provinces = re.findall(r'[\u4e00-\u9fa5]+', province_list)
        time.sleep(1)
        province_dropdown = driver.find_elements(By.CLASS_NAME, "ant-select-selection--single")[0]
        province_dropdown.click()
        time.sleep(2)
        province_index = 1
        for province in provinces:
            try:
                # 选择省份
                province_dropdown = driver.find_elements(By.CLASS_NAME, "ant-select-selection--single")[0]
                province_dropdown.click()
                province_xpath = f'//*[@id="proline"]/div/div[1]/div/div[1]/div[2]/div/div/div/div[{province_index}]'
                province_option = WebDriverWait(driver, 5).until(EC.element_to_be_clickable((By.XPATH, province_xpath)))
                province_option.click()
                province_index = province_index + 1
                # 遍历年份
                # 获取年份
                year_dropdown = driver.find_elements(By.CLASS_NAME, "ant-select-selection--single")[1]
                year_dropdown.click()
                year_opt=WebDriverWait(driver,5).until(EC.element_to_be_clickable((By.XPATH,'//*[@id="proline"]/div/div[1]/div/div[3]/div[2]/div/div')))
                years_str = \
                    driver.find_elements(By.XPATH, '//*[@id="proline"]/div/div[1]/div/div[3]/div[2]/div/div')[
                        0].text

                years = years_str.split('\n')
                time.sleep(1)
                year_dropdown = driver.find_elements(By.CLASS_NAME, "ant-select-selection--single")[1]
                year_dropdown.click()
                time.sleep(2)
                print(years)
                for year in years:
                    try:
                        # 选择年份
                        year_dropdown = driver.find_elements(By.CLASS_NAME, "ant-select-selection--single")[1]
                        year_dropdown.click()

                        year_xpath = f"//li[contains(@class, 'ant-select-dropdown-menu-item') and text()='{year}']"
                        year_option = WebDriverWait(driver, 5).until(
                            EC.element_to_be_clickable((By.XPATH, year_xpath))
                        )
                        year_option.click()
                        time.sleep(1)

                        # 获取科目类型
                        subject_dropdown = driver.find_elements(By.CLASS_NAME, "ant-select-selection--single")[
                            2]
                        subject_dropdown.click()
                        sunject_opt=WebDriverWait(driver,5).until(EC.element_to_be_clickable((By.XPATH,'//*[@id="proline"]/div/div[1]/div/div[5]/div[2]/div/div')))

                        type_list = \
                            driver.find_elements(By.XPATH,
                                                 '//*[@id="proline"]/div/div[1]/div/div[5]/div[2]/div/div')[
                                0].text
                        subjects = get_subject_types(type_list)
                        subject_dropdown = driver.find_elements(By.CLASS_NAME, "ant-select-selection--single")[2]
                        subject_dropdown.click()
                        time.sleep(2)
                        print(subjects)
                        # 遍历科目类型
                        for subject in subjects:
                            try:
                                print(f"处理：{province} - {year} - {subject}")

                                subject_dropdown =  driver.find_elements(By.CLASS_NAME, "ant-select-selection--single")[2]
                                subject_dropdown.click()
                                subject_option = WebDriverWait(driver, 5).until(
                                    EC.presence_of_element_located(
                                        (By.XPATH, f"//li[contains(text(), '{subject}')]"))
                                )
                                subject_option.click()


                                # 重新打开下拉菜单以供下次选择

                                # 获取分数线数据
                                score_opt=WebDriverWait(driver,5).until(EC.visibility_of_all_elements_located((By.XPATH,'//*[@id="proline"]/div/div[2]/table/tbody')))
                                time.sleep(5)
                                score_elements = driver.find_elements(By.XPATH,
                                                                      '//*[@id="proline"]/div/div[2]/table/tbody')
                                for score_element in score_elements:
                                    score_data = score_element.text.split('\n')
                                    print(f"分数线数据：{score_data}")
                                    save_score_data(school_name,school_id, province, year, subject, score_data)
                            except Exception as e:
                                print(f"处理科目 {subject} 时出错：{str(e)}")
                                save_failed_data(school_name,school_id, province, year, subject)
                                continue

                    except Exception as e:
                        print(f"处理年份 {year} 时出错：{str(e)}")
                        save_failed_data(school_name,school_id, province, year,"")
                        continue

            except Exception as e:
                print(f"处理省份 {province} 时出错：{str(e)}")
                save_failed_data(school_name,school_id, province, "", "")
                continue
        driver.quit()
    except Exception as e:
        print(f"程序执行出错：{str(e)}")




import pickle

def main():
    try:
        with open('school_data.pkl', 'rb') as file:
            schools = pickle.load(file)


        # 使用线程池执行多线程爬取，此处如果没有设置代理服务器则建议并行个数在5以内，防止封IP


        with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:
            # 提交所有学校任务到线程池
            futures = []
            for school in schools:
                futures.append(executor.submit(process_school, school))
                time.sleep(60)

            # 等待所有任务完成
            for future in concurrent.futures.as_completed(futures):
                try:
                    future.result()  # 获取任务结果或异常
                except Exception as e:
                    print(f"线程执行错误: {str(e)}")


    finally:

        print("\n程序执行完毕")


if __name__ == "__main__":
    main()